# -*- coding: utf-8 -*-

'''
Created on 09-07-10

@author: peterd
'''


from datetime import datetime
from sgmllib import SGMLParser


from geo.models import *
from lbc.models import *
from infra.logging import logger
from web.service.GeoCodingBroker import GeoCodingBroker
from infra.lexical.CharMapper import cp1250_to_ascii


class LBCParser(SGMLParser):
	'''
	classdocs
	'''

	def __init__(self, lbcategory, dataProcessor, parsedUrl = None):
		SGMLParser.__init__(self)		
		
		self.lbcategory_ = lbcategory
		self.dataProcessor_ = dataProcessor
		self.parsedUrl_ = parsedUrl
		
		self.errors_ = []
		self.city_ = None
		self.street_ = None
		self.streetNum_ = None
		self.building_ = ""
		self.region_ = None
		self.numOrgsTot_ = 0
		self.numOrgs_ = 0
		self.organizations_ = []
		self.cancelled_ = False
		self.corId_ = None
		
		self.nextPage_ = None
		
		self.homeUrl_ = None
		self.curOrg_ = None
		self.image_ = None
		
		
	def setLBCategory(self, lbcategory):
		self.lbcategory_ = lbcategory
		

	def setDataProcessor(self, dataProcessor):
		self.dataProcessor_ = dataProcessor
		
		

	def setParsedUrl(self, parsedUrl):
		self.parsedUrl_ = parsedUrl
		
	def reset__(self):
		del self.errors_[:]
		self.street_ = None
		self.streetNum_ = None
		self.building_ = ""
		self.cancelled_ = False
		self.corId_ = None
		self.homeUrl_ = None
		self.image_ = None
		
	def preParse(self):
		pass
		


	def parse(self, htmlPage, dataSource, preParse = True):
		self.region_ = dataSource.region
		self.city_ = dataSource.city
		self.provider_ = dataSource.broker.provider
		
		if preParse:
			self.preParse()
			
		self.numOrgs_ = 0
		del self.organizations_[:]
			
		
		log = logger.getLogger()
		filteredContent = self.filter(htmlPage)
		log.info("Parsing html content ...")
		self.feed(self.encode(filteredContent))
		
		self.close()
		log.info("Content parsed.")
		
		if self.numOrgs_ == 0:
			log.warning('Parser with id = "%s", lbcategory = "%s", parsedUrl = "%s" did not find any organization, revision might be needed.' % (self.getId(), cp1250_to_ascii(self.lbcategory_.name), self.parsedUrl_))
		else:
			log.info('Parser with id = "%s" found %d organizations of which %d are new (total = %s).' % (self.getId(), self.numOrgs_, len(self.organizations_), self.numOrgsTot_))		
			
		return self.organizations_
	
	
	
	def handle_data(self, data):
		if not self.cancelled_:
			data = data.strip()
			data = data.replace('\r\n', '')
			data = data.replace('\n', '')
			
			if len(data) > 0:
				self.do_handle_data(self.decode(data))


	def filter(self, content):		
		filteredContent = re.sub('<!-\w+->', '', content)
		filteredContent = re.sub('\r\n', '', filteredContent)
		pat = re.compile('<([S|s]cript|SCRIPT).*?<\/([S|s]cript|SCRIPT)>', re.DOTALL)
		filteredContent = pat.sub('', filteredContent)
			
		return filteredContent

	# REVISIT: make sure the encoding won't strip & from org title
	def encode(self, content):
		content = content.replace('&', '&amp;')
		return content
	
	
	def decode(self, content):
		content = content.replace('&amp;', '&')
		content = content.replace('&quot;', '"')
		return content
		

	def cancel(self):
		self.cancelled_ = True
		

	def getErrors(self):
		return self.errors_
	
	
	def setCity(self, data):
		log = logger.getLogger()
		
		try:
			self.city_ = findCity(data, self)
			log.debug('city = {%s}' % self.city_)
		except Exception, e:
			raise e
		
		
	def getCity(self):
		return self.city_
			

	def setStreet(self, data):
		log = logger.getLogger()
		
		try:
			self.street_, self.streetNum_ = findStreet(data, self.city_, self)			
			log.debug('street = {%s}, number = %s' % (self.street_, self.streetNum_))			
		except Exception, e:
			raise e
		
		
	def setHomeUrl(self, url):
		if self.curOrg_ and self.isUrlValid__(url):
			self.curOrg_.home_url = url
			self.curOrg_.save()


	def setImage(self, image):
		if self.curOrg_:
			self.curOrg_.image = image
			self.curOrg_.save()



	def setBuilding(self, building):
		self.building_ = building
		
	# TODO: encapsulate parameters as instances variables, use reset__() before repopulating
	def createOrganization(self, name, email = None, url = None, phone = None, logo = None):
		log = logger.getLogger()
		location = None
				
		try:
			if self.building_:
				for org in Organization.objects.filter(building = self.building_):
					if org.location.city == self.city_ and org.location.street == self.street_:
						location = org.location
						if org.location.number != self.streetNum_:
							log.info('Mapped location = {city = {%s}, street = {%s}, number = "%s", building = "%s"} to location = {%s}' % (self.city_, self.street_, self.streetNum_, cp1250_to_ascii(self.building_), location)) 
					break
			if not location:
				location = findLocation(self.city_, self.street_, self.streetNum_, self.building_, self)
		except Exception, e:
			raise e
		
		if len(name) > MAX_ORG_NAME_LEN:
			name = name[0:MAX_ORG_NAME_LEN - 3] + '...'
		
		org = None						
		organizations = Organization.objects.filter(name = name).filter(location = location).filter(provider = self.provider_)
		if len(organizations) == 0 and url:
			organizations = Organization.objects.filter(url = url).filter(location = location).filter(provider = self.provider_)
		if len(organizations) == 0:
			orgStatus = OrganizationStatus.objects.get(code = OrganizationStatus.VISIBLE)
			if not orgStatus:
				log.error('Failed to obtain OrganizationStatus.VISIBLE instance.')
			else:			
				org = Organization()
				org.category = self.lbcategory_
				org.name = name
				org.location = location
				org.building = self.building_
				org.email = email
				org.url = url
				org.phone = phone
				org.logo = logo
				org.provider = self.provider_
				org.status = orgStatus
				org.cor_id = self.corId_
				if self.isUrlValid__(self.homeUrl_):
					org.home_url = self.homeUrl_
				
				log.debug('created organization = {%s}' % org)
				self.organizations_.append(org)
		else:
			org = organizations[0]
		
		org.image = self.image_
		org.last_fetch = datetime.today()
		org.save()
			
		self.numOrgs_ += 1
		self.numOrgsTot_ += 1
		
		self.curOrg_ = org

		return org
	
	
	def onUnknownCity(self, cityName):
		log = logger.getLogger()
		msg = '%s\n' % ''.join(self.errors_)		
		events = Event.objects.filter(event_msg = msg)
		eventType = EventType.objects.get(code = EventType.UNKNOWN_CITY)
		eventStatus = EventStatus.objects.get(code = EventStatus.PENDING)
		if not eventType:
			log.error('Failed to obtain EventType.UNKNOWN_CITY instance.')
		elif not eventStatus:
			log.error('Failed to obtain EventStatus.PENDING instance.')
		elif len(events) == 0:
			event = Event()
			event.event_date = datetime.today()
			event.event_type = eventType
			event.event_msg = msg
			event.event_status = eventStatus
			event.provider = self.provider_.name
			event.lbc_path = self.lbcategory_.toFullPath()
			event.sender = self.getId()
			event.url = self.parsedUrl_
			event.city_ascii_name = cityName				
			event.save()
			
		return None


	def onMultipleCities(self, cityName):
		log = logger.getLogger()
		msg = '%s\n' % ''.join(self.errors_)		
		events = Event.objects.filter(event_msg = msg)
		eventType = EventType.objects.get(code = EventType.MULTIPLE_CITIES)
		eventStatus = EventStatus.objects.get(code = EventStatus.PENDING)
		if not eventType:
			log.error('Failed to obtain EventType.MULTIPLE_CITIES instance.')
		elif not eventStatus:
			log.error('Failed to obtain EventStatus.PENDING instance.')
		elif len(events) == 0:
			event = Event()
			event.event_date = datetime.today()
			event.event_type = eventType
			event.event_msg = msg
			event.event_status = eventStatus
			event.lbc_path = self.lbcategory_.toFullPath()
			event.provider = self.provider_.name
			event.sender = self.getId()
			event.url = self.parsedUrl_
			event.city_ascii_name = cityName				
			event.save()
			
		return None


	def onUnknownStreet(self, streetName):
		street = None
		streetAsciiName = cp1250_to_ascii(streetName)
		
		geoCoder = GeoCodingBroker()	
		(lat, lng, accuracy) = geoCoder.geoCode(streetAsciiName, self.streetNum_, self.city_.ascii_name)
		districts = District.objects.filter(name = 'None')
		if lat and lng and accuracy >= GEO_STREET_LEVEL_ACCURACY and len(districts) > 0:
			street = Street()
			street.name = streetName
			street.ascii_name = streetAsciiName 
			street.city = self.city_
			street.district = districts[0]
			street.save()
		
		else:
			log = logger.getLogger()
			msg = '%s\n' % ''.join(self.errors_)		
			events = Event.objects.filter(event_msg = msg)
			eventType = EventType.objects.get(code = EventType.UNKNOWN_STREET)
			eventStatus = EventStatus.objects.get(code = EventStatus.PENDING)
			if not eventType:
				log.error('Failed to obtain EventType.UNKNOWN instance.')
			elif not eventStatus:
				log.error('Failed to obtain EventStatus.PENDING instance.')
			elif len(events) == 0:
				event = Event()
				event.event_date = datetime.today()
				event.event_type = eventType
				event.event_msg = msg
				event.event_status = eventStatus
				event.provider = self.provider_.name
				event.lbc_path = self.lbcategory_.toFullPath()
				event.sender = self.getId()
				event.url = self.parsedUrl_
				if self.city_:
					event.city_ascii_name = self.city_.ascii_name
				event.street_ascii_name = streetName				
				event.save()

		return street
	

	def onMultipleStreets(self, streetName):
		log = logger.getLogger()
		msg = '%s\n' % ''.join(self.errors_)		
		events = Event.objects.filter(event_msg = msg)
		eventType = EventType.objects.get(code = EventType.MULTIPLE_STREETS)
		eventStatus = EventStatus.objects.get(code = EventStatus.PENDING)
		if not eventType:
			log.error('Failed to obtain EventType.MULTIPLE_STREETS instance.')
		elif not eventStatus:
			log.error('Failed to obtain EventStatus.PENDING instance.')
		elif len(events) == 0:
			event = Event()
			event.event_date = datetime.today()
			event.event_type = eventType
			event.event_msg = msg
			event.event_status = eventStatus
			event.provider = self.provider_.name
			event.lbc_path = self.lbcategory_.toFullPath()
			event.sender = self.getId()
			event.url = self.parsedUrl_
			if self.city_:
				event.city_ascii_name = self.city_.ascii_name
			event.street_ascii_name = streetName				
			event.save()
			
		return None

	
	def onUnknownLocation(self, cityName, streetName, \
						  streetNum = None, building = None):
		log = logger.getLogger()
		location = None
				
		geoCoder = GeoCodingBroker()	
		(lat, lng, accuracy) = geoCoder.geoCode(self.street_.ascii_name, self.streetNum_, self.city_.ascii_name)
		if lat and lng and accuracy >= GEO_STREET_LEVEL_ACCURACY:
			location = Location()
			location.street = self.street_
			location.number = self.streetNum_
			location.city = self.city_
			location.lat = lat
			location.lng = lng
			location.accuracy = accuracy
			location.save()
			
			return location
		
					
		if not streetNum:
			streetNum = ''
		if not building:
			building = ''

		msg = '%s\n' % ''.join(self.errors_)
		
		locationMap = LocationMap.objects.filter(lbc_path = self.lbcategory_.toFullPath(), \
												 src_city_ascii_name = cityName, \
												 src_street_ascii_name = streetName, \
												 src_street_number = streetNum,
												 src_building_name = cp1250_to_ascii(building))
												
		if len(locationMap) > 0:
			try:
				self.setCity(locationMap[0].dst_city_ascii_name)
				self.setStreet('%s %s' % (locationMap[0].dst_street_ascii_name, locationMap[0].dst_street_number))
				location = findLocation(self.city_, self.street_, self.streetNum_, locationMap[0].dst_building_name)
				
				resolveEvent(eventType = EventType.UNKNOWN_LOCATION, 
							lbcPath = self.lbcategory_.toFullPath(), 
							msgSender = self.getId(), 
							eventMsg = msg)									
			except Exception, e:
				log.error('Failed to obtain destination location using location map id = %s, reason: %s' % (locationMap[0].id, e))

		if not location:			
			events = Event.objects.filter(lbc_path = self.lbcategory_.toFullPath(), \
										   sender = self.getId(), \
										   event_msg = msg)
			eventType = EventType.objects.get(code = EventType.UNKNOWN_LOCATION)
			eventStatus = EventStatus.objects.get(code = EventStatus.PENDING)
			if not eventType:
				log.error('Failed to obtain EventType.UNKNOWN_LOCATION instance.')
			elif not eventStatus:
				log.error('Failed to obtain EventStatus.PENDING instance.')
			elif len(events) == 0:
				event = Event()
				event.event_date = datetime.today()
				event.event_type = eventType
				event.event_msg = msg
				event.event_status = eventStatus
				event.provider = self.provider_.name
				event.lbc_path = self.lbcategory_.toFullPath()
				event.sender = self.getId()
				event.url = self.parsedUrl_
				if self.city_:
					event.city_ascii_name = self.city_.ascii_name
				if self.street_:
					event.street_ascii_name = self.street_.ascii_name
				event.street_number = self.streetNum_			
				if self.building_:
					event.building_name = self.building_
				
				event.save()
				
		return location
	
			
	
		
	def onKnownLocation(self, cityName, streetName, \
						  streetNum = None, building = None):
		try:
			resolveEvent(eventType = EventType.UNKNOWN_LOCATION, 
						 lbcPath = self.lbcategory_.toFullPath(),
						 msgSender = self.getId(),
						 cityAsciiName = cityName,
						 streetAsciiName = streetName, 
					     streetNumber = streetNum, 
					     buildingName = building)		
		except Exception:
			pass


	def onKnownStreet(self, streetName):
		try:
			resolveEvent(eventType = EventType.UNKNOWN_STREET, 
						 lbcPath = self.lbcategory_.toFullPath(),
						 msgSender = self.getId(),
						 streetAsciiName = streetName) 
		except Exception:
			pass
			
			
	def onKnownCity(self, cityName):
		try:
			resolveEvent(eventType = EventType.UNKNOWN_CITY, 
						 lbcPath = self.lbcategory_.toFullPath(),
						 msgSender = self.getId(),
						 cityAsciiName = cityName) 
		except Exception:
			pass


	def hasNextPage(self):
		return self.nextPage_ != None						


	def getNextPage(self):
		return self.nextPage_
	
	
	def isUrlValid__(self, url):
		return url and len(url) and url.strip() != 'http://'